from mxnet import autograd, gluon, init, nd
from mxnet.gluon import data as gdata, loss as gloss, nn
import numpy as np
import pandas as pd
import time
from fbprophet import Prophet
#import pystan
import matplotlib.pyplot as plt
from pandas.core.arrays.sparse import dtype

def read_data(path):
    train_data=pd.read_csv(path+'train.csv',dtype={'is_pro':object},encoding="utf-8")
    test_data=pd.read_csv(path+'test.csv',dtype={'is_pro':object},encoding="utf-8")
    date_ch=pd.read_csv(path+'date_ch.csv',dtype={'is_pro':object},encoding="utf-8")
    goods_ch=pd.read_csv(path+'goods_ch.csv',dtype={'is_pro':object},encoding="utf-8")
    del date_ch['official_holiday_name']#有official_holiday_id代替
    del date_ch['festival_name']#有festival_id代替
    del goods_ch['season_class_name']#有season_class_id代替
    del goods_ch['div_name']#有div_id代替
    return train_data,test_data,date_ch,goods_ch

def type_change(raw_train,raw_test,date_ch):
    raw_train['sale_date']=raw_train['sale_date'].astype('object')#转换train数据格式
    raw_train['goodsid']=raw_train['goodsid'].astype('object')
    raw_train['year_code']=raw_train['year_code'].astype('object')
    raw_train['quanter_code']=raw_train['quanter_code'].astype('object')
    raw_train['month_code']=raw_train['month_code'].astype('object')
    raw_train['day_week_cn']=raw_train['day_week_cn'].astype('object')
    raw_train['week_day_code']=raw_train['week_day_code'].astype('object')
    raw_train['div_id']=raw_train['div_id'].astype('object')
    raw_train['catg_l_id']=raw_train['catg_l_id'].astype('object')
    raw_train['catg_m_id']=raw_train['catg_m_id'].astype('object')
    raw_train['catg_s_id']=raw_train['catg_s_id'].astype('object')
    raw_train['season_class']=raw_train['season_class'].astype('object')

    raw_test['sale_date']=raw_test['sale_date'].astype('object')#转换test数据格式
    raw_test['goodsid']=raw_test['goodsid'].astype('object')
    raw_test['year_code']=raw_test['year_code'].astype('object')
    raw_test['quanter_code']=raw_test['quanter_code'].astype('object')
    raw_test['month_code']=raw_test['month_code'].astype('object')
    raw_test['day_week_cn']=raw_test['day_week_cn'].astype('object')
    raw_test['week_day_code']=raw_test['week_day_code'].astype('object')
    raw_test['div_id']=raw_test['div_id'].astype('object')
    raw_test['catg_l_id']=raw_test['catg_l_id'].astype('object')
    raw_test['catg_m_id']=raw_test['catg_m_id'].astype('object')
    raw_test['catg_s_id']=raw_test['catg_s_id'].astype('object')
    raw_test['season_class']=raw_test['season_class'].astype('object')

    date_ch['sale_date']=date_ch['sale_date'].astype('object')

def merge_attr(train_data,test_data,date_ch,goods_ch):
    raw_train=pd.merge(train_data,date_ch,how='inner',on='sale_date')#合并训练集属性
    raw_train=pd.merge(raw_train,goods_ch,how='inner',on='goodsid')
    raw_test=pd.merge(test_data,date_ch,how='inner',on='sale_date')#合并测试集属性
    raw_test=pd.merge(raw_test,goods_ch,how='inner',on='goodsid')
    return raw_train,raw_test

def main():
    begin = time.time()
    
    '''读入数据'''
    path="G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/预测组赛题/预测组赛题/"
    train_data,test_data,date_ch,goods_ch=read_data(path)
    type_change(train_data,test_data,date_ch)#类型转换,区分ohject和数值型,方便归一化
    print('读入数据完成......')
    print(train_data.dtypes,date_ch.dtypes)
    '''将date和goods文件的属性添加到train和test中'''
    raw_train,raw_test=merge_attr(train_data,test_data,date_ch,goods_ch)#合并属性
    type_change(raw_train,raw_test)
    print('属性合并完成......')

    '''合并raw_train和raw_test中所有的属性并进行数据处理'''
    train_cols=list(range(3,6))+list(range(7,16))+[20]#label是第3列属性,所以需要取1-2，4-22列属性
    test_cols=list(range(2,5))+list(range(6,15))+[19]
    all_features=pd.concat((raw_train.iloc[:,train_cols], raw_test.iloc[:,test_cols]))
    #连续数据归一化
    numeric_features=all_features.dtypes[all_features.dtypes!='object'].index#取所有数值型变量的列名
    all_features[numeric_features]=all_features[numeric_features].apply(lambda x:(x-x.mean())/(x.std()))#将数据化为服从标准正态分布
    all_features[numeric_features]=all_features[numeric_features].fillna(0)#用均值0取代缺失值
    
    attr_list=['year_code', 'quanter_code', 'season_code', 'month_code',
    'day_week_cn','is_weekend','official_holiday_code', 'festival_code', 'season_class']
    all_features=pd.get_dummies(all_features,dummy_na=True,prefix=attr_list,columns=attr_list,dtype=int)#扩增属性
    print('数据处理完成......')

    '''将raw_train和raw_test还原成train和test两个数据集'''
    n_train=raw_train.shape[0]
    print(all_features.shape)
    #train_features=nd.array(all_features[:n_train].values)
    #test_features=nd.array(all_features[n_train:].values)
    #train_labels=nd.array(raw_train.sales_qty.values).reshape((-1, 1))
    #all_features[:n_train].to_csv('G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/train_features.csv')
    #all_features[n_train:].to_csv('G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/test_features.csv')
    print(test_features)
    print('导出文件完成......')
    finish=time.time()
    print("用时:{}".format(finish-begin))

'''
def prophet_test():
    path='G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/预测组赛题/预测组赛题/'
    train_data=pd.read_csv(path+'train.csv',usecols=[0,2],nrows=639,encoding="utf-8")
    #test_data=pd.read_csv(path+'test.csv',dtype={'is_pro':object},encoding="utf-8")
    train_data['sale_date']=train_data['sale_date'].apply(pd.to_datetime)
    #print(train_data.iloc[0:4])
    print(train_data.dtypes)
    print(len(train_data))
    train_data.columns=['ds','y']
    train_data.set_index('ds').y.plot()
    #print(train_data.iloc[0:639])
    prophet=Prophet()
    prophet.fit(train_data)
    future = prophet.make_future_dataframe(periods=365)
    print(future.tail())
    forecast = prophet.predict(future)
    print(forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].head())
    fig1=prophet.plot(forecast)
    prophet.plot_components(forecast).savefig('fig1.png')
'''

if __name__=='__main__':
    main()
